#include "WebPageQuery.h"
#include "WebPage.h"
#include <cassert>
#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
#include <string>
#include <utility>
#include <unistd.h>
#include <string.h>
#include <regex>
#include <vector>
#include <unicode/unistr.h>
#include <unicode/translit.h>
using namespace std;


void WebPageQuery::iniSearchCon(){
    readInvert();
    readOffset();
    readPages();
}


void WebPageQuery::readInvert(){
    ifstream ifs("../../data/invertIndex.dat");
    if(!ifs.good()){
        cout << "Open invertIndex.dat error!" << endl;
        exit(1); 
    }
    string line; 
    while(getline(ifs, line, '\n')){
        if(line.empty()){
            continue;
        }
        else{
            istringstream iss(line);
            string word;
            int id;
            double power;
            iss >> word;
            while(iss >> id){
                iss >> power; 
                _invertIndexLib.insert(make_pair(word, make_pair(id, power))); 
            }
        } 
    }
    cout << "读取 invertIndex.dat 完成!" << endl;
    ifs.close();
}


void WebPageQuery::readOffset(){
    ifstream ifs("../../data/offset.dat");
    if(!ifs.good()){
        cout << "Open offset.dat error!" << endl;
        exit(1); 
    }
    string line; 
    while(getline(ifs, line, '\n')){
        if(line.empty()){
            continue;
        }
        else{
            istringstream iss(line);
            int id, begin, end;
            iss >> id;
            iss >> begin;
            iss >> end;
            _offsetlib.insert(make_pair(id, make_pair(begin, end))); 
            iss.clear();
        } 
    }
    cout << "读取 offset.dat 完成!" << endl;
    ifs.close();
}


void WebPageQuery::readPages(){
    ifstream ifs("../../data/ripepage.dat");
    if(!ifs.good()){
        cout << "Open offset.dat error!" << endl;
        exit(1); 
    }
    for(auto &offCon:_offsetlib){
        char buff[65535];
        ifs.seekg(offCon.second.first);
        ifs.read(buff, offCon.second.second);
        WebPage web,webc;
        getWebPageCon(web,webc,buff);
        _pages.push_back(web);
        _pagesCpe.push_back(webc);
    }
    cout << "读取 ripepage.dat 完成!" << endl;
    ifs.close();
}


void WebPageQuery::getWebPageCon(WebPage& web, WebPage& webc,char* buff){
    regex pattern;
    smatch match;
    string objMatch(buff);
    
    pattern.assign("<title>(.*?)</title>");
    regex_search(objMatch, match, pattern);
    web.title = match.str(1); // 提取的内容
    webc.title = match.str(1); // 提取的内容
    

    pattern.assign("<link>(.*?)</link>");
    regex_search(objMatch, match, pattern);
    web.link = match.str(1); 
    webc.link = match.str(1); // 提取的内容
    
    // 匹配 description 
    pattern.assign("<description>");
    if(regex_search(objMatch, match, pattern)){
        string shortdes;
        int peekBegin = match.position(0);  
        pattern.assign("</description>");
        regex_search(objMatch, match, pattern);
        int peekEnd = match.position(0);
        webc.description = objMatch.substr(peekBegin + 13, peekEnd - peekBegin - 13); 
        if((peekEnd - peekBegin - 13) < 400){
            shortdes = objMatch.substr(peekBegin + 13, peekEnd - peekBegin - 13); 
        }
        else{
            shortdes = objMatch.substr(peekBegin + 13, 400); 
        }
        icuFix(shortdes, web);
    }
    else{
        web.description = ""; 
        webc.description = ""; 
    }
 
    // 匹配 content
    pattern.assign("<content>");
    if(regex_search(objMatch, match, pattern)){
        int peekBegin = match.position(0);  
        pattern.assign("</content>");
        regex_search(objMatch, match, pattern);
        int peekEnd = match.position(0);
        webc.content = objMatch.substr(peekBegin + 9, peekEnd - peekBegin - 9); 
        web.content = objMatch.substr(peekBegin + 9, peekEnd - peekBegin - 9); 
    }
    else{
        web.content = ""; 
        webc.content = ""; 
    }
}


// 切除不完整的 UTF-8 字符，以便 json 使用
void WebPageQuery::icuFix(string shortdes, WebPage& web){
    icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(shortdes);
    // 获取字符串长度
    int32_t length = ustr.length();
    // 检查最后一个字符是否为完整的 UTF-8 字符
    UChar32 ch = ustr.char32At(length - 1);
    if (U8_IS_SINGLE(ch) || U8_IS_LEAD(ch)) {}
    else{
        ustr.remove(length - 1, 1);
        length--;
    }
    ustr.toUTF8String(web.description); 
}



// unordered_multimap<string, pair<int, double>> _invertIndexLib;
vector<WebPage> WebPageQuery::doQuery(string key){
    vector<WebPage> queryWeb;
    vector<string> words = _wordCutTool.cut(key);
    map<int, pair<int, vector<double>>> countMap; // 文章id, 出现次数, 权重 

    // 统计搜索的词语在哪些文章出现过
    // 如果文章词语出现的次数与查询词的个数相等 
    // 说名该文章为词语的合集 
    for(auto word:words){
        // 查找与给定词相同的键，得到出现该次的文章id, 权重 
        auto queryPages = _invertIndexLib.equal_range(word);
        for(auto it = queryPages.first; it != queryPages.second; ++it){
            // it 是查找到的键值对 
            if(countMap.count(it->second.first)){
                countMap[it->second.first].first += 1; 
                countMap[it->second.first].second.push_back(it->second.second); 
            }
            else{
                countMap[it->second.first].first = 1; 
                countMap[it->second.first].second.push_back(it->second.second); 
            }
        }
    }

    // vector<int> sortList = similaritySort(countMap, words.size()); 

    // 不走算法
    for(auto it:countMap){ 
        if(it.second.first == words.size()){
            queryWeb.push_back(_pages[it.first - 1]);
        } 
    }

    if(queryWeb.size() == 0){
        cout << "没有匹配的文章" << endl;     
    }
    return queryWeb;
}

// width 维数
vector<int> WebPageQuery::similaritySort(map<int, pair<int, vector<double>>> &countMap, int width ){
    vector<pair<int, vector<double>>> sortList;
    vector<int> resultList; // 结果变量
    for(auto it:countMap){ // 查找对应维数的文章
        if(it.second.first == width){
            //得到文章 id, 权重数组, 并组合成待排序列表, 以第一个元素为基准 
            sortList.push_back(make_pair(it.first,it.second.second));
        }
    }
    map<double, int> result;   
    for(int i = 1; i < sortList.size();i++){
        result.insert(make_pair(calCosSimilarity(sortList[0].second, sortList[i].second), sortList[i].first));
    }
    resultList.push_back(sortList[0].first);
    for(auto elem:result){
        resultList.push_back(elem.second);
    } 
    return resultList;
}   


double WebPageQuery::calMagnitude(const vector<double>& vec) {
    double sum = 0.0;
    for (double value : vec) {
        sum += value * value;
    }
    return sqrt(sum);
}


// 计算两个向量的余弦相似度
double WebPageQuery::calCosSimilarity(const vector<double>& vec1, const vector<double>& vec2) {
    // 检查向量的维度是否一致
    if (vec1.size() != vec2.size()) {
        return 0.0;
    }

    double dotProduct = 0.0;
    for (int i = 0; i < vec1.size() ; i++) {
        dotProduct += vec1[i] * vec2[i];
    }

    double magnitude1 = calMagnitude(vec1);
    double magnitude2 = calMagnitude(vec2);
    return dotProduct / (magnitude1 * magnitude2);
}


